import numpy as np
import pandas as pd
import scipy.sparse
from sklearn.decomposition import PCA,NMF,SparsePCA,TruncatedSVD,LatentDirichletAllocation
import scipy.stats as sps
from scipy.sparse import vstack

seed = 1024
np.random.seed(seed)

path = '../data/'

train = pd.read_pickle(path + "train_context_tfidf.pkl")
valid = pd.read_pickle(path + "valid_context_tfidf.pkl")
dev = pd.read_pickle(path+"dev_context_tfidf.pkl")

data_all = vstack((train,valid,dev)).T

svd = TruncatedSVD(n_components=12,random_state=1123)
svd.fit(data_all)
svd_fea = svd.components_.T

train_tfidf = svd_fea[:train.shape[0]]
valid_tfidf = svd_fea[train.shape[0]:(train.shape[0]+valid.shape[0])]
dev_tfidf = svd_fea[(train.shape[0]+valid.shape[0]):]

f = 'svd'
pd.to_pickle(train_tfidf, path + 'train_%s_tfidf.pkl' % f)
pd.to_pickle(valid_tfidf,path+'valid_%s_tfidf.pkl'%f)
pd.to_pickle(dev_tfidf, path + 'dev_%s_tfidf.pkl' % f)